package com.light.dp.spark.config;

import com.light.dp.spark.kryo.CustomKryoRegistrator;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaSparkContext;
import org.springframework.context.annotation.Bean;
import org.springframework.stereotype.Component;

/**
 * @author fuhui
 * @Description：
 * @data 2025/1/8 17:29
 */
@Component
public class SparkConfig {
    @Bean
    public SparkConf sparkConf() {
        SparkConf conf = new SparkConf()
                .setAppName("DatProcessingSparkDemo")
                .set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
                .set("spark.kryo.registrator", "com.light.dp.spark.kryo.CustomKryoRegistrator")
                .set("spark.kryo.registrationRequired","true")
//                .setMaster("spark://runa-dev-72:7077");  // 指定spark的master节点
                .setMaster("local[*]");  // 设置为本地模式，使用所有可用核心，可根据实际情况调整为集群模式等
        return conf;
    }

    @Bean
    public JavaSparkContext javaSparkContext(SparkConf sparkConf) {
        return new JavaSparkContext(sparkConf);
    }
}